# -*- coding: utf-8 -*-

from scaws import AWS


DATA_PATH = 'data'
PREPARE_FOR_DIRS = ['log']

CFG = {
    'sleep_time': 6,
    'run_sleep_time': 3,
    # 'force_run_once': True,
    's3_bucket': 'c-news',
    'components': [
        {
            'name': 'conveyor',
            'class_name': 'scpy2.components.aws_s3sqs.S3Sqs',
            'cfg': {
                'aws': AWS,
                'queue_used_for_receive': 'smoke_test_task',
                'receive_s3_enabled': False,
                # 'queue_used_for_send': 'smoke_test_downloaded',
                'receive_args': {
                    'MaxNumberOfMessages': 1,
                    'WaitTimeSeconds': 6
                }
            }
        },
        {
            'name': 'reaper',
            'class_name': 'scpy2.components.requests_session.RequestsSession',
            'cfg': {
                'proxies': {
                    'http': 'http://127.0.0.1:8001',
                    # 'https': 'socks5://127.0.0.1:8087'
                }
            }
        },
    ],
    'crawl_processors': {
        'toutiao': 'toutiao_news.TouTiaoNews',
    }
}
